<!DOCTYPE HTML>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" /> 
    <title> - 天地维杰网</title>
    <meta name="keywords" content="系统架构,shutdown,不与天斗,Domino,博客,程序员,架构师,笔记,技术,分享,java,Redis">
    
    <meta property="og:title" content="">
    <meta property="og:site_name" content="天地维杰网">
    <meta property="og:image" content="/img/author.jpg"> 
    <meta name="title" content=" - 天地维杰网" />
    <meta name="description" content="天地维杰网 | 博客 | 软件 | 架构 | Java "> 
    <link rel="shortcut icon" href="http://www.shutdown.cn/img/favicon.ico" />
    <link rel="apple-touch-icon" href="http://www.shutdown.cn/img/apple-touch-icon.png" />
    <link rel="apple-touch-icon-precomposed" href="http://www.shutdown.cn/img/apple-touch-icon.png" />
    <link href="http://www.shutdown.cn/js/vendor/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/js/vendor/fancybox/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/css/main.css" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/css/syntax.css" rel="stylesheet" type="text/css" />
    <script type="text/javascript" id="hexo.configuration">
  var NexT = window.NexT || {};
  var CONFIG = {
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post"},
     fancybox: true, 
    motion: true
  };
</script>
</head>
<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">
<div class="container one-collumn sidebar-position-left page-home  ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"> <div class="site-meta  custom-logo ">

  <div class="custom-logo-site-title">
    <a href="http://www.shutdown.cn"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">天地维杰网</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle">人如秋鸿来有信，事若春梦了无痕</p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
    <ul id="menu" class="menu">
      
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />首页
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/redis/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-battery-full"></i> <br />Redis
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/java/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-coffee"></i> <br />java
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/linux/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-linux"></i> <br />linux
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/daily/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-bug"></i> <br />日常问题
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/spring/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-child"></i> <br />Spring和Springboot
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/spring/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-fire"></i> <br />Mac相关
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/middleware/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-gavel"></i> <br />中间件
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/jiagou/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-rocket"></i> <br />架构
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/python/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-ship"></i> <br />python
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/front/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-bolt"></i> <br />前端
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/jvm/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-balance-scale"></i> <br />jvm
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/post/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />归档
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/about/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />关于
          </a>
        </li>
      
      <li class="menu-item menu-item-search">
        <a href="javascript:;" class="popup-trigger"> <i class="menu-item-icon fa fa-search fa-fw"></i> <br /> 搜索</a>
      </li>
    </ul>
    <div class="site-search">
      <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>

    </div>
</nav>

 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            
<section id="posts" class="posts-expand">
  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
    <header class="post-header">
      <h1 class="post-title" itemprop="name headline">
        <a class="post-title-link" href="http://www.shutdown.cn/post/linux%E8%BF%90%E7%BB%B4%E5%9F%BA%E7%A1%80%E9%87%87%E9%9B%86%E9%A1%B9/" itemprop="url">
        
        </a>
      </h1>
      <div class="post-meta">
      <span class="post-time">
<span class="post-meta-item-icon">
    <i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">时间：</span>
<time itemprop="dateCreated" datetime="2016-03-22T13:04:35+08:00" content="0001-01-01">
    0001-01-01
</time>
</span> 
      
      
       <span>
&nbsp; | &nbsp;
<span class="post-meta-item-icon">
    <i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读：</span>
<span class="leancloud-visitors-count">397 字 ~2分钟</span>
</span>
      </div>
    </header>
    <div class="post-body" itemprop="articleBody">
    

    

<h1 id="1-linux运维基础采集项">1. Linux运维基础采集项</h1>

<p>做运维，不怕出问题，怕的是出了问题，抓不到现场，两眼摸黑。所以，依靠强大的监控系统，收集尽可能多的指标，意义重大。但哪些指标才是有意义的呢，本着从实践中来的思想，各位工程师在长期摸爬滚打中总结出来的经验最有价值。</p>

<p>在各位运维工程师长期的工作实践中，我们总结了在系统运维过程中，经常会参考的一些指标，主要包括以下几个类别：</p>

<ul>
<li>CPU</li>
<li>Load</li>
<li>内存</li>
<li>磁盘</li>
<li>IO</li>
<li>网络相关</li>
<li>内核参数</li>
<li>ss 统计输出</li>
<li>端口采集</li>
<li>核心服务的进程存活信息采集</li>
<li>关键业务进程资源消耗</li>
<li>NTP offset采集</li>
<li>DNS解析采集</li>
</ul>

<p>每个类别，具体的详细指标如下，这些指标，都是open-falcon的agent组件直接支持的。falcon-agent每隔一定时间间隔（目前是60秒）会采集一次相关的指标，并汇报给server端。</p>

<h1 id="2-cpu相关采集项">2. CPU相关采集项</h1>

<p>计算方法：通过采集/proc/stat来得到，大家可以参考sar命令的统计输出来理解。</p>

<ul>
<li>cpu.idle：Percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request.</li>
<li>cpu.busy：与cpu.idle相对，他的值等于100减去cpu.idle。</li>
<li>cpu.guest：Percentage of time spent by the CPU or CPUs to run a virtual processor.</li>
<li>cpu.iowait：Percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request.</li>
<li>cpu.irq：Percentage of time spent by the CPU or CPUs to service hardware interrupts.</li>
<li>cpu.softirq：Percentage of time spent by the CPU or CPUs to service software interrupts.</li>
<li>cpu.nice：Percentage of CPU utilization that occurred while executing at the user level with nice priority.</li>
<li>cpu.steal：Percentage of time spent in involuntary wait by the virtual CPU or CPUs while the hypervisor was servicing another virtual processor.</li>
<li>cpu.system：Percentage of CPU utilization that occurred while executing at the system level (kernel).</li>
<li>cpu.user：Percentage of CPU utilization that occurred while executing at the user level (application).</li>
<li>cpu.cnt：cpu核数。</li>
<li>cpu.switches：cpu上下文切换次数，计数器类型。</li>
</ul>

<h1 id="3-磁盘相关采集项">3. 磁盘相关采集项</h1>

<p>计算方法：先读取/proc/mounts拿到所有挂载点，然后通过syscall.Statfs_t拿到blocks和inode的使用情况。每个metric都会附加一组tag描述，类似mount=$mount,fstype=$fstype，其中$mount是挂载点，比如/home，$fstype是文件系统，比如ext4。</p>

<ul>
<li>df.bytes.free：磁盘可用量，int64</li>
<li>df.bytes.free.percent：磁盘可用量占总量的百分比，float64，比如32.1</li>
<li>df.bytes.total：磁盘总大小，int64</li>
<li>df.bytes.used：磁盘已用大小，int64</li>
<li>df.bytes.used.percent：磁盘已用大小占总量的百分比，float64</li>
<li>df.inodes.total：inode总数，int64</li>
<li>df.inodes.free：可用inode数目，int64</li>
<li>df.inodes.free.percent：可用inode占比，float64</li>
<li>df.inodes.used：已用的inode数据，int64</li>
<li>df.inodes.used.percent：已用inode占比，float64</li>
</ul>

<h1 id="4-megacli工具输出">4. megacli工具输出</h1>

<p>使用 megacli 工具读取 RAID 相关信息，每个metric都会附件一组tag描述，用来标明所属PD或者 VD，PD格式为PD=Enclosure_ID:SLOT_ID，比如PD=32:0表明第一块磁盘 ，VD=0 表明第一个逻辑磁盘。</p>

<ul>
<li>sys.disk.lsiraid.pd.Media_Error_Count：这个及以下三个指标目前仅作为数据收集，不一定意味磁盘损坏（只是表示损坏概率变大）</li>
<li>sys.disk.lsiraid.pd.Other_Error_Count</li>
<li>sys.disk.lsiraid.pd.Predictive_Failure_Count</li>
<li>sys.disk.lsiraid.pd.Drive_Temperature</li>
<li>sys.disk.lsiraid.pd.Firmware_state：如果值不为0，则此物理磁盘出现问题</li>
<li>sys.disk.lsiraid.vd.cache_policy：如果值不为0，表示此逻辑磁盘缓存策略和设置不符</li>
<li>sys.disk.lsiraid.vd.state： 如果值不为0，表示此逻辑磁盘出现问题</li>
</ul>

<h1 id="5-smart工具输出">5. SMART工具输出</h1>

<p>使用 smartctl 工具读取磁盘 SMART 信息，目前所有指标仅作为数据收集，不一定意味磁盘损坏（只是表示概率变大），每个metric都会有一组tag描述，表明盘符，例如device=/dev/sda。</p>

<ul>
<li>sys.disk.smart.Reallocated_Sector_Ct</li>
<li>sys.disk.smart.Spin_Retry_Count</li>
<li>sys.disk.smart.Reallocated_Event_Count</li>
<li>sys.disk.smart.Current_Pending_Sector</li>
<li>sys.disk.smart.Offline_Uncorrectable</li>
<li>sys.disk.smart.Temperature_Celsius</li>
</ul>

<h1 id="6-分区读写监控">6. 分区读写监控</h1>

<p>测试所有已挂载分区是否可读写，每个metric都会有一组tag描述，表示挂载点，比如mount=/home</p>

<ul>
<li>sys.disk.rw： 如果值不为0，表明此分区读写出现问题</li>
</ul>

<h1 id="7-io相关采集项">7. IO相关采集项</h1>

<p>计算方法：每秒采集一次/proc/diskstats，计算差值，都是计数器类型的。每个metric都会有一组tag描述，形如device=$device，用来表示具体的设备，比如sda1、sdb。用户可以参考iostat的帮助文档来理解具体的metric含义。</p>

<ul>
<li>disk.io.ios_in_progress：Number of actual I/O requests currently in flight.</li>
<li>disk.io.msec_read：Total number of ms spent by all reads.</li>
<li>disk.io.msec_total：Amount of time during which ios_in_progress &gt;= 1.</li>
<li>disk.io.msec_weighted_total：Measure of recent I/O completion time and backlog.</li>
<li>disk.io.msec_write：Total number of ms spent by all writes.</li>
<li>disk.io.read_merged：Adjacent read requests merged in a single req.</li>
<li>disk.io.read_requests：Total number of reads completed successfully.</li>
<li>disk.io.read_sectors：Total number of sectors read successfully.</li>
<li>disk.io.write_merged：Adjacent write requests merged in a single req.</li>
<li>disk.io.write_requests：total number of writes completed successfully.</li>
<li>disk.io.write_sectors：total number of sectors written successfully.</li>
<li>disk.io.read_bytes：单位是byte的数字</li>
<li>disk.io.write_bytes：单位是byte的数字</li>
<li>disk.io.avgrq_sz：下面几个值就是iostat -x 1看到的值</li>
<li>disk.io.avgqu-sz</li>
<li>disk.io.await</li>
<li>disk.io.svctm</li>
<li>disk.io.util：是个百分数，比如56.43，表示56.43%</li>
</ul>

<h1 id="8-机器负载相关采集项">8. 机器负载相关采集项</h1>

<p>计算方法：读取/proc/loadavg，都是原始值类型的：</p>

<ul>
<li>load.1min</li>
<li>load.5min</li>
<li>load.15min</li>
</ul>

<h1 id="9-内存相关采集项">9. 内存相关采集项</h1>

<p>计算方法：读取/proc/meminfo 中的内容，其中的mem.memfree是free+buffers+cached，mem.memused=mem.memtotal-mem.memfree。用户具体可以参考free命令的输出和帮助文档来理解每个metric的含义。</p>

<ul>
<li>mem.memtotal：内存总大小</li>
<li>mem.memused：使用了多少内存</li>
<li>mem.memused.percent：使用的内存占比</li>
<li>mem.memfree</li>
<li>mem.memfree.percent</li>
<li>mem.swaptotal：swap总大小</li>
<li>mem.swapused：使用了多少swap</li>
<li>mem.swapused.percent：使用的swap的占比</li>
<li>mem.swapfree</li>
<li>mem.swapfree.percent</li>
</ul>

<h1 id="10-网络相关采集项">10. 网络相关采集项</h1>

<p>计算方法：读取/proc/net/dev的内容，每个metric都附加有一组tag，形如iface=$iface，标明具体那个interface，比如eth0。metric中带有in的表示流入情况，out表示流出情况，total是总量in+out，支持的metric如下：</p>

<ul>
<li>net.if.in.bytes</li>
<li>net.if.in.compressed</li>
<li>net.if.in.dropped</li>
<li>net.if.in.errors</li>
<li>net.if.in.fifo.errs</li>
<li>net.if.in.frame.errs</li>
<li>net.if.in.multicast</li>
<li>net.if.in.packets</li>
<li>net.if.out.bytes</li>
<li>net.if.out.carrier.errs</li>
<li>net.if.out.collisions</li>
<li>net.if.out.compressed</li>
<li>net.if.out.dropped</li>
<li>net.if.out.errors</li>
<li>net.if.out.fifo.errs</li>
<li>net.if.out.packets</li>
<li>net.if.total.bytes</li>
<li>net.if.total.dropped</li>
<li>net.if.total.errors</li>
<li>net.if.total.packets</li>
</ul>

<h1 id="11-端口采集项">11. 端口采集项</h1>

<p>计算方法，通过ss -ln，来判断指定的端口是否处于listen状态。原始值类型，值要么是1：代表在监听，要么是0，代表没有在监听。每个metric都附件一组tag，形如port=$port，$port就是具体的端口。</p>

<ul>
<li>net.port.listen</li>
</ul>

<h1 id="12-机器内核配置">12. 机器内核配置</h1>

<ul>
<li>kernel.maxfiles： 读取的/proc/sys/fs/file-max</li>
<li>kernel.files.allocated：读取的/proc/sys/fs/file-nr第一个Field</li>
<li>kernel.files.left：值=kernel.maxfiles-kernel.files.allocated</li>
<li>kernel.maxproc：读取的/proc/sys/kernel/pid_max</li>
</ul>

<h1 id="13-ntp采集项">13. ntp采集项</h1>

<p>使用 ntpq -pn 获取本机时间相对于 ntp 服务器的 offset。</p>

<ul>
<li>sys.ntp.offset： 本机偏移时间，单位为ms，值过大或者为0则表明有异常，需要报警</li>
</ul>

<h1 id="14-进程监控">14. 进程监控</h1>

<ul>
<li>proc.num：判断某个进程的数目，这里需要分两个场景，一种是根据进程的名字来判定，比如name=sshd；另外一种是根据cmdline来判定，比如Java的应用进程名可能都是java，根据第一种情况没法做区分，此时可以配置cmdline，如cmdline=./falcon_agent-c./cfg.ini</li>
</ul>

<h1 id="15-进程资源监控">15. 进程资源监控</h1>

<ul>
<li>process.cpu.all：进程和它的子进程使用的sys+user的cpu，单位是jiffies</li>
<li>process.cpu.sys：进程和它的子进程使用的sys cpu，单位是jiffies</li>
<li>process.cpu.user：进程和它的子进程使用的user cpu，单位是jiffies</li>
<li>process.swap：进程和它的子进程使用的swap，单位是page</li>
<li>process.fd：进程使用的文件描述符个数</li>
<li>process.mem：进程占用内存，单位byte</li>
</ul>

<h1 id="16-ss命令输出">16. ss命令输出</h1>

<ul>
<li>ss.orphaned</li>
<li>ss.closed</li>
<li>ss.timewait</li>
<li>ss.slabinfo.timewait</li>
<li>ss.synrecv</li>
<li>ss.estab</li>
</ul>

    </div>
    <footer class="post-footer">
     

     <div class="post-nav">
    <div class="post-nav-next post-nav-item">
    
        <a href="http://www.shutdown.cn/post/linux%E6%9F%A5%E7%9C%8B%E7%89%A9%E7%90%86cpu%E4%B8%AA%E6%95%B0%E6%A0%B8%E6%95%B0%E9%80%BB%E8%BE%91cpu%E4%B8%AA%E6%95%B0/" rel="next" title="">
        <i class="fa fa-chevron-left"></i> 
        </a>
    
    </div>

    <div class="post-nav-prev post-nav-item">
    
        <a href="http://www.shutdown.cn/post/linux%E8%BF%9B%E7%A8%8B%E7%BA%BF%E7%A8%8B%E4%B8%8Ecpu%E7%9A%84%E4%BA%B2%E5%92%8C%E6%80%A7affinity/" rel="prev" title="">
         <i class="fa fa-chevron-right"></i>
        </a>
    
    </div>
</div>
      
     
     
     






    </footer>
  </article>
</section>

          </div>
        </div>
        <div class="sidebar-toggle">
  <div class="sidebar-toggle-line-wrap">
    <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
    <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
    <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
  </div>
</div>
<aside id="sidebar" class="sidebar">
  <div class="sidebar-inner">

    <section class="site-overview sidebar-panel  sidebar-panel-active ">
      <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image"
        src="http://www.shutdown.cn/img/author.jpg"
        alt="不与天斗Domino" />
    <p class="site-author-name" itemprop="name">不与天斗Domino</p>
    <p class="site-description motion-element" itemprop="description"> 
        Programmer &amp; Architect</p>
</div>
      <nav class="site-state motion-element">
    <div class="site-state-item site-state-posts">
      <a href="http://www.shutdown.cn/post/">
        <span class="site-state-item-count">169</span>
        <span class="site-state-item-name">日志</span>
      </a>
    </div>
    <div class="site-state-item site-state-categories">    
        <a href="http://www.shutdown.cn/categories/">      
         
        <span class="site-state-item-count">10</span>
        
        <span class="site-state-item-name">分类</span>
        
        </a>
    </div>

    <div class="site-state-item site-state-tags">
        <a href="http://www.shutdown.cn/tags/">
         
        <span class="site-state-item-count">131</span>
        
        <span class="site-state-item-name">标签</span>
        </a>
    </div>
</nav>
      
      

      

      <div class="links-of-blogroll motion-element inline">
<script type="text/javascript" src="//rf.revolvermaps.com/0/0/8.js?i=&amp;m=0&amp;s=220&amp;c=ff0000&amp;cr1=ffffff&amp;f=arial&amp;l=33&amp;bv=35" async="async"></script>
</div>

    </section>
    
  </div>
</aside>

      </div>
    </main>
   
    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  <span itemprop="copyrightYear">  &copy; 
  2013 - 2022</span>
  <span class="with-love"><i class="fa fa-heart"></i></span>
  <span class="author" itemprop="copyrightHolder">天地维杰网</span>
  <span class="icp" itemprop="copyrightHolder"><a href="https://beian.miit.gov.cn/" target="_blank">京ICP备13019191号-1</a></span>
</div>
<div class="powered-by">
  Powered by - <a class="theme-link" href="http://gohugo.io" target="_blank" title="hugo" >Hugo v0.63.2</a>
</div>
<div class="theme-info">
  Theme by - <a class="theme-link" href="https://github.com/xtfly/hugo-theme-next" target="_blank"> NexT
  </a>
</div>


      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span id="scrollpercent"><span>0</span>%</span>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/jquery/index.js?v=2.1.3"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/fastclick/lib/fastclick.min.js?v=1.0.6"></script> 
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/velocity/velocity.min.js?v=1.2.1"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/velocity/velocity.ui.min.js?v=1.2.1"></script>
<script src="http://www.shutdown.cn/js/vendor/ua-parser-js/dist/ua-parser.min.js?v=0.7.9"></script>

<script src="http://www.shutdown.cn/js/vendor/fancybox/jquery.fancybox.pack.js?v=2.1.5"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/utils.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/motion.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/affix.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/schemes/pisces.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/scrollspy.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/post-details.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/toc.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/bootstrap.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/search.js"></script>
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({
    extensions: ["tex2jax.js"],
    jax: ["input/TeX", "output/HTML-CSS"],
    tex2jax: {
      inlineMath: [ ['$','$'] ],
      displayMath: [ ['$$','$$'] ],
      processEscapes: true
    },
    "HTML-CSS": { fonts: ["TeX"] }
  });
</script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_HTMLorMML' async></script>
</body>
</html>